#python
from os import listdir
from os.path import join
from collections import defaultdict
from re import sub, search, findall
#src
from .util import *
from .log import *
from .errors import *
from .configuration import config
#third-party
import cardsharp as cs
__all__ = ['Report']
#mapping to determine unique values in a log file
#log type: list of variable names
_log_unique_key = {
0: ['convert_state', 'convert_key'],
1: ['rapstatex', 'convert_state', 'ori', 'fed'],
2: ['convert_state', 'convert_key_values'],
3: ['convert_state', 'convert_key', 'extracted_cts'],
90: ['bad_value', 'table_name', 'rule_loc'],
99: ['variable', 'error_message'],
}
def _dedupe(log_data, log_type):
unique_rows = []
for row in log_data:
if log_type in [77,78,91]:
break
#if the key based on type, if log_type not listed raise error
try:
key = [row[var] for var in _log_unique_key[log_type]]
except:
raise ReportError('Unrecognized log type')
if key not in unique_rows:
unique_rows.append(key)
else:
row.delete()
[docs]class Report(object):
"""Class to create/update CCHR reports based on data contained in the log files.
The reports are stored in the web interface database. The tables
contain one row of data for each unique log entry per data run,
where a unique entry is determined by a key of:
segment_id + variable_id + phase_id + region_id + key_1 + key_2 + key_3.
The Report class is called when during pre_process, process, and transform
phases to generate reports based on the log output from each of the
respective phases.
The reports can also be generated as excel/text files.
Public Methods:
...
"""
def __init__(self, report_path, **kw):
self.log = Log(**kw)
self.report_path = report_path
self.verbose = kw.get('verbose')
[docs] def create_reports(self):
for log_file in listdir(self.log.log_dir):
if is_file(r'%s/%s' % (self.log.log_dir, log_file)):
if self.verbose:
print 'creating %s report' % log_file.replace('.txt', '')
try:
log_ds, type = self.log.load(log_file)
log_ds.wait()
_dedupe(log_ds, type)
save_format = 'spss' if len(log_ds.rows) > 65000 else 'excel'
out_file = 'report_%s' % sub('_\d+', '', log_file).replace('.txt', FORMAT_INFO[save_format])
log_ds.save(source=join(self.report_path, out_file), format=save_format, overwrite=True)
cs.wait()
#create prediction report
if 'ncic' in out_file and 'wvfbi' not in out_file:
cs.wait()
'''convert_key_names convert_key_values predictions
cstatnumx|cchglitx |cocaine possess-w/int to distribute (federal) |||9|9|99|9'''
#create one prediction report for every unique set of key names
_keys = defaultdict(list)
for row in log_ds:
_keys[row['convert_key_names']].append((row['convert_key_values'].split('|'), row['predictions'].split('|')))
for c, key in enumerate(_keys.iterkeys()):
_vars = ['key:%s' % k for k in key.split('|')]
_vars.extend(['value:%s' % v for v in ['ncic', 'chg', 'inc', 'dom', 'weap', 'cdv', 'pg']])
ds = cs.Dataset(_vars)
for value in _keys[key]:
_val = [v if v else None for v in value[0]]
_val.extend([v if v else None for v in value[1]])
ds.add_row(_val)
for var in ds.variables:
if 'key' in var.name: continue
ds.variables[var.name].convert('integer')
ds.save(source=join(self.report_path, out_file), format=save_format, overwrite=True, dataset='predict_%i' % c)
cs.wait()
except LogError as e:
print 'skipping report for %s. %s' % (log_file, e)
cs.wait()